import re
import requests
from jsonpath import jsonpath
import time
'''
使用的是requests模块对考研帮App数据进行抓取，由于请求过多，请求速度过慢，我们使用scrapy_redis框架分布式进行数据的抓取

'''
url = 'https://mapi.kyboye.com/news/index?psize=10&skip=0&type=1&schid=&speid=20115100&_t=1588579494355'


def get_url(url):
    '''
    请求url返回json响应
    :param url:
    :return:
    '''
    headers = {'Connection': 'Keep-Alive',
               'Accept-Encoding': 'gzip',
               'Cookie': 'FDX_auth=1325f4fb08CSx5Kl9%2BMj7LhbY2kBLJxBWHsoQQEqrZPyAEORYJ4fQQskQAZxyzMyE70SsSFNrqKEYILFC%2FHoiYIw; FDX_sid=w23565396_b58fe4c3506aa8e932aa70',
               'KY-APPVER': '3.4.1',
               'KY-APPCHG': 'ky_360',
               'KY-APPTYPE': '2',
               'KY-SPEID': '20115100',
               'KY-APPVERS': '96',
               'KY-SYSVER': '5.1.1',
               'KY-SCHID': '1033',
               'KY-YEAR': '2021',
               'KY-SYSDEV': 'samsung+SM-N960F',
               'KY-TOKEN': 'ff30493f5c39982f38a316b137713bda',
               'KY-UUID': '8be44003623e3edc5dc5480f9f65caee',
               'User-Agent': 'KaoYanBang AipBot/1.0 (KaoYanClub-Android/3.4.1; android/5.1.1; samsung+SM-N960F)'}
    response = requests.get(url=url, headers=headers, verify=False).json()
    return response


def get_category():
    '''
    抓取分类名称与分类id
    :return:
    '''
    url = 'https://mapi.kyboye.com/news/index?psize=10&skip=0&type=1&schid=&speid=20115100&_t=1588579494355'
    resp = get_url(url)
    name = jsonpath(resp, '$.res.cate[*].name')
    id = jsonpath(resp, '$.res.cate[*].id')
    category_list = []
    for i in range(len(name)):
        category_dict = {
            'name': name[i],
            'id': id[i]
        }
        category_list.append(category_dict)
    return category_list


def timestamp():
    '''
    生成时间戳
    :return:
    '''
    t = int(time.time() * 1000)
    return t


def fabu_time(t):
    '''
    将时间戳转换成时间格式
    :param t:
    :return:
    '''
    timeArray = time.localtime(t)
    fabu_time = time.strftime("%Y-%m-%d %H:%M:%S", timeArray)
    return fabu_time


def list_news(data):
    '''
    获取新闻列表页中的所有新闻列表信息
    :param data:
    :return:
    '''
    id = jsonpath(data, '$.res.list[*].id')
    return id


def new_detail(data):
    '''
    提取新闻详情数据
    :param data:
    :return:
    '''
    ctime = jsonpath(data, '$.res.ctime')[0]
    time = fabu_time(int(ctime))
    title = ''.join(jsonpath(data, '$.res.title'))
    new_dict = {
        'title': title,
        'time': time,
        'copyfrom': jsonpath(data, '$.res.copyfrom')[0],
        'content': jsonpath(data, '$.res.content')
    }
    return new_dict


def wipe_out(string):
    pre = re.compile('>(.*?)<')
    s1 = ''.join(pre.findall(string))
    s2 = s1.replace('。', '。\n').replace(' ', '')
    return s2


def save_new(new_dict, dirname):
    try:
        filename = new_dict['title'].replace('“', '').replace('”', '').replace('/', '-').replace(':', '-')
        content = ''.join(new_dict['content'])
        con = wipe_out(content)
        string = new_dict['title'] + '\n' + new_dict['time'] + '      ' + new_dict['copyfrom'] + '\n' + con
        with open(r'D:/code/考研帮app抓取/新闻/{}/'.format(dirname) + filename + '.txt', 'a', encoding='utf-8') as f:
            f.write(string)
    except:
        pass


def run():
    '''
    主程序
    :return:
    '''
    cate_list = get_category()
    for cate in cate_list:
        cid = cate['id']
        c_name = cate['name']
        if c_name == '报考经验':
            urlstring = 'https://mapi.kyboye.com/news/article/list?cid={}&sid=1033&skip={}&wd=&scid=0&speid=20115100&psize=20&type=1&_t={}'
            whi_run(urlstring, cid, c_name)

        elif c_name == '政策新闻':
            urlstring = 'https://mapi.kyboye.com/news/article/list?cid=201&sid=1033&skip=0&wd=&scid={}&speid=20115100&psize=20&type=1&_t={}'
            whi_run(urlstring, cid, c_name)

        elif c_name == '考研心路':
            urlstring = 'https://mapi.kyboye.com/news/article/list?cid={}&sid=1033&skip={}&wd=&scid=0&speid=20115100&psize=20&type=1&_t={}'
            whi_run(urlstring, cid, c_name)

        elif c_name == '公共课复习':
            urlstring = 'https://mapi.kyboye.com/news/article/list?cid={}&sid=1033&skip={}&wd=&scid=0&speid=20115100&psize=20&type=1&_t={}'
            whi_run(urlstring, cid, c_name)
        elif c_name == '复试调剂':
            urlstring = 'https://mapi.kyboye.com/news/article/list?cid={}&sid=1033&skip={}&wd=&scid=0&speid=20115100&psize=20&type=1&_t={}'
            whi_run(urlstring, cid, c_name)

        else:
            pass


def whi_run(urlstring, cid, filename):
    '''
    代码抽取，将循环进行数据抓取保存的代码抽取出来，解决代码的冗余
    :param urlstring:
    :param cid:
    :param filename:
    :return:
    '''
    skip = 0
    while True:
        try:
            t = timestamp()
            url = urlstring.format(
                cid, skip, t)
            resp = get_url(url)
            new_list = list_news(resp)
            for new_id in new_list:
                t = timestamp()
                url = 'https://mapi.kyboye.com/news/article/get?id={}&psize=10&skip=0&_t={}'.format(new_id, t)
                resp = get_url(url)
                new_dict = new_detail(resp)
                save_new(new_dict, filename)
            skip += 20
        except:
            break


if __name__ == '__main__':
    run()
